﻿/* ----------------------------------------------------------------------------
 * Copyright (c) Philip Stears 
 *
 * This source code is subject to terms and conditions of the Microsoft Public 
 * License (Ms-PL).
 * 
 * By using this source code in any fashion, you are agreeing to be bound 
 * by the terms of the Microsoft Public License.
 * 
 * To request a copy of the Microsoft Public License, send a request by email
 * to license@connectbasic.net.
 *
 * This notice must not be removed from this software.
 * ---------------------------------------------------------------------------- */

namespace ConnectBasic.Execution
{
    using System;
    using System.Collections.Generic;
    using System.Text;
    using Microsoft.Scripting;
    using Microsoft.Scripting.Runtime;

    /// <summary>
    /// Implements the ConnectBasic tokenizer.
    /// </summary>
    /// <devdoc>
    /// This class is internal because at some point it would be nice
    /// to implement XML literal support. Now, if we do that, we can't
    /// tokenize the input stream directly - the parser needs to tell
    /// the tokenizer whether it should apply token rules for XML or BASIC
    /// contexts. If we need to add tool support we'll need to implement a
    /// "token walker" which walks the parse tree and returns tokens that way.
    /// </devdoc>
    internal class BasicTokenizer
    {
        private const int TOKENIZER_BUFFER_SIZE = 4096;
        private static IDictionary<string, BasicTokenKind> sKeywords = CreateKeywords();

        // Tokenizer state
        private SourceUnit mSource;
        private TokenizerBuffer mBuffer;
        private ErrorSink mSink;

        // Current token state
        private BasicToken mCurrent = new BasicToken();

        #region .cctor

        private static IDictionary<string, BasicTokenKind> CreateKeywords()
        {
            var result = new Dictionary<string, BasicTokenKind>(StringComparer.OrdinalIgnoreCase);

            result.Add("Public", BasicTokenKind.KeywordPublic);
            result.Add("Private", BasicTokenKind.KeywordPrivate);
            result.Add("Stop", BasicTokenKind.KeywordStop);
            result.Add("Function", BasicTokenKind.KeywordFunction);
            result.Add("Sub", BasicTokenKind.KeywordSub);
            result.Add("Exit", BasicTokenKind.KeywordExit);
            result.Add("End", BasicTokenKind.KeywordEnd);
            result.Add("While", BasicTokenKind.KeywordWhile);
            result.Add("Wend", BasicTokenKind.KeywordWend);
            result.Add("For", BasicTokenKind.KeywordFor);
            result.Add("Next", BasicTokenKind.KeywordNext);
            result.Add("Dim", BasicTokenKind.KeywordDim);
            result.Add("As", BasicTokenKind.KeywordAs);
            result.Add("GoTo", BasicTokenKind.KeywordGoTo);
            result.Add("Error", BasicTokenKind.KeywordError);
            result.Add("On", BasicTokenKind.KeywordOn);
            result.Add("Resume", BasicTokenKind.KeywordResume);
            result.Add("ByRef", BasicTokenKind.KeywordByRef);
            result.Add("ByVal", BasicTokenKind.KeywordByVal);
            result.Add("Option", BasicTokenKind.KeywordOption);
            result.Add("Explicit", BasicTokenKind.KeywordExplicit);
            result.Add("Strict", BasicTokenKind.KeywordStrict);
            result.Add("Compare", BasicTokenKind.KeywordCompare);
            result.Add("Text", BasicTokenKind.KeywordText);
            result.Add("Binary", BasicTokenKind.KeywordBinary);
            result.Add("Const", BasicTokenKind.KeywordConst);
            result.Add("If", BasicTokenKind.KeywordIf);
            result.Add("Then", BasicTokenKind.KeywordThen);
            result.Add("ElseIf", BasicTokenKind.KeywordElseIf);
            result.Add("Else", BasicTokenKind.KeywordElse);
            result.Add("Optional", BasicTokenKind.KeywordOptional);
            result.Add("Is", BasicTokenKind.KeywordIs);
            result.Add("IsNot", BasicTokenKind.KeywordIsNot);
            result.Add("Nothing", BasicTokenKind.KeywordNothing);
            result.Add("Set", BasicTokenKind.KeywordSet);
            result.Add("Call", BasicTokenKind.KeywordCall);
            result.Add("Not", BasicTokenKind.KeywordNot);
            result.Add("To", BasicTokenKind.KeywordTo);
            result.Add("Step", BasicTokenKind.KeywordStep);
            result.Add("Do", BasicTokenKind.KeywordDo);
            result.Add("Until", BasicTokenKind.KeywordUntil);
            result.Add("Loop", BasicTokenKind.KeywordLoop);

            return result;
        }

        #endregion

        #region .ctor

        internal BasicTokenizer(SourceUnit source, ErrorSink sink)
        {
            if (source == null) throw new ArgumentNullException("source");
            mSource = source;
            mSink = sink ?? ErrorSink.Null;
            mBuffer = new TokenizerBuffer(source.GetReader(), SourceLocation.MinValue, TOKENIZER_BUFFER_SIZE, true);
        }

        #endregion

        #region Public Methods + Properties

        /// <summary>
        /// Gets the error sink.
        /// </summary>
        public ErrorSink ErrorSink
        {
            get
            {
                return mSink;
            }
        }

        /// <summary>
        /// Reads the next BASIC token.
        /// </summary>
        /// <returns>
        /// True if a token was successfully read, false if the end of the stream was hit, or an invalid
        /// token was reached.
        /// </returns>
        public BasicTokenKind ReadBasicNext()
        {

            // Discard the previous token and mark the start 
            // of a new one (even if we've only just started)
            mBuffer.DiscardToken();

            int ch = mBuffer.Read();

            // Okay, what've we got?
            switch (ch)
            {
                case -1:
                    this.SetTokenEof();
                    break;

                case '_':

                    // Begin scope to isolate variables
                    {
                        int next = mBuffer.Peek();

                        if (IsIdentifierChar(next))
                        {
                            this.ReadIdentifierOrKeyword();
                        }
                        else
                        {
                            this.SetTokenOperator(BasicTokenKind.LineContinuation);
                        }

                        break;
                    }

                case '0':
                case '1':
                case '2':
                case '3':
                case '4':
                case '5':
                case '6':
                case '7':
                case '8':
                case '9':
                    this.ReadNumber();
                    break;

				case ':':

					{
						// This could be a : or a := so check the next character
						int next = mBuffer.Peek();

						if (next == '=')
						{
							mBuffer.Read();
							this.SetTokenOperator(BasicTokenKind.ParameterName);
						}
						else
						{
							this.SetTokenOperator(BasicTokenKind.Colon);
						}
						
						break;

					}

                case '"':
                    this.ReadString();
                    break;

                case '\'':
                    this.ReadComment();
                    break;

                case '.':
                    this.SetTokenOperator(BasicTokenKind.Dot);
                    break;

                case ',':
                    this.SetTokenOperator(BasicTokenKind.Comma);
                    break;

                case '^':
                    this.SetTokenOperator(BasicTokenKind.Power);
                    break;

                case '&':

                    // Begin scope to isolate variables
                    {
                        int next = mBuffer.Peek();

                        if (next == 'H' || next == 'h')
                        {
                            mBuffer.Read();
                            this.ReadHex();
                        }
                        else
                        {
                            this.SetTokenOperator(BasicTokenKind.Concatenate);
                        }

                        break;
                    }

                case '*':
                    this.SetTokenOperator(BasicTokenKind.Multiply);
                    break;

                case '(':
                    this.SetTokenOperator(BasicTokenKind.ParanthesisOpen);
                    break;

                case ')':
                    this.SetTokenOperator(BasicTokenKind.ParanthesisClose);
                    break;

                case '-':
                    this.SetTokenOperator(BasicTokenKind.Subtract);
                    break;

                case '+':
                    this.SetTokenOperator(BasicTokenKind.Add);
                    break;

                case '=':
                    this.SetTokenOperator(BasicTokenKind.AssignOrEqual);
                    break;

                case '\\':
                    this.SetTokenOperator(BasicTokenKind.DivideIntegral);
                    break;

                case '/':
                    this.SetTokenOperator(BasicTokenKind.DivideFloatingPoint);
                    break;

                case '<':

                    // Begin scope to isolate variables
                    {
                        int next = mBuffer.Peek();

                        if (next == '=')
                        {
                            mBuffer.Read();
                            this.SetTokenOperator(BasicTokenKind.LessThanOrEqualTo);
                        }
                        else if (next == '>')
                        {
                            mBuffer.Read();
                            this.SetTokenOperator(BasicTokenKind.NotEqualTo);
                        }
                        else
                        {
                            this.SetTokenOperator(BasicTokenKind.LessThan);
                        }

                        break;
                    }

                case '>':

                    // Begin scope to isolate variables
                    {
                        int next = mBuffer.Peek();

                        if (next == '=')
                        {
                            mBuffer.Read();
                            this.SetTokenOperator(BasicTokenKind.LessThanOrEqualTo);
                        }
                        else
                        {
                            this.SetTokenOperator(BasicTokenKind.LessThan);
                        }

                        break;
                    }

                default:

                    if (char.IsLetter((char)ch))
                    {
                        this.ReadIdentifierOrKeyword();
                    }
                    else if (IsWhitespace((char)ch))
                    {
                        this.ReadWhitespace();
                    }
                    else if (IsNewLine((char)ch))
                    {
                        this.ReadNewLines();
                    }
                    else
                    {
                        this.SetTokenUnexpectedCharacter();
                    }

                    break;
            }

            // Return the type of whatever token we got
            return mCurrent.Kind;
        }

        /// <summary>
        /// Gets the current token.
        /// </summary>
        public BasicToken CurrentToken
        {
            get { return mCurrent; }
        }

        #endregion

        #region Private Methods - Token Read

        private void ReadWhitespace()
        {
            int next = mBuffer.Peek();

            while (IsWhitespace(next))
            {
                mBuffer.Read();
                next = mBuffer.Peek();
            }

            mBuffer.MarkSingleLineTokenEnd();
            mCurrent.Category = TokenCategory.WhiteSpace;
            mCurrent.Kind = BasicTokenKind.Whitespace;
            mCurrent.Span = mBuffer.TokenSpan;
            mCurrent.ValueString = mBuffer.GetTokenString();
        }

        private void ReadNewLines()
        {
            int next = mBuffer.Peek();

            while (IsNewLine(next))
            {
                mBuffer.Read();
                next = mBuffer.Peek();
            }

            mBuffer.MarkMultiLineTokenEnd();
            mCurrent.Category = TokenCategory.None;
            mCurrent.Kind = BasicTokenKind.NewLines;
            mCurrent.Span = mBuffer.TokenSpan;
            mCurrent.ValueString = mBuffer.GetTokenString();
        }

        private void ReadHex()
        {
            throw new NotImplementedException();
        }

        private void ReadComment()
        {

            // Clear existing literal data
            this.SetTokenHelper_ClearLiteralData();

            // Read to the end of the line and set state     
            mBuffer.ReadLine();
            mBuffer.MarkSingleLineTokenEnd();

            mCurrent.Category = TokenCategory.LineComment;
            mCurrent.Kind = BasicTokenKind.Comment;
            mCurrent.Span = mBuffer.TokenSpan;
            mCurrent.ValueString = mBuffer.GetTokenString();
        }

        private void ReadIdentifierOrKeyword()
        {

            // Read until we hit a non-identifier character
            int next = mBuffer.Peek();

            while (IsIdentifierChar(next))
            {
                mBuffer.Read();
                next = mBuffer.Peek();
            }

            mBuffer.MarkSingleLineTokenEnd();
            mCurrent.Span = mBuffer.TokenSpan;
            mCurrent.ValueString = mBuffer.GetTokenString();

            // Is it a keyword?
            BasicTokenKind result;

            if (sKeywords.TryGetValue(mCurrent.ValueString, out result))
            {
                mCurrent.Kind = result;
                mCurrent.Category = TokenCategory.Keyword;
            }
            else if ("true".Equals(mCurrent.ValueString, StringComparison.OrdinalIgnoreCase))
            {
                mCurrent.Kind = BasicTokenKind.LiteralBoolean;
                mCurrent.ValueBoolean = true;
                mCurrent.Category = TokenCategory.Keyword;
            }
            else if ("false".Equals(mCurrent.ValueString, StringComparison.OrdinalIgnoreCase))
            {
                mCurrent.Kind = BasicTokenKind.LiteralBoolean;
                mCurrent.ValueBoolean = false;
                mCurrent.Category = TokenCategory.Keyword;
            }
            else
            {
                mCurrent.Kind = BasicTokenKind.Identifier;
                mCurrent.Category = TokenCategory.Identifier;
            }
        }

        private void ReadNumber()
        {

            // Read until we hit a non-numeric character
            int next = mBuffer.Peek();

            while (next >= '0' && next <= '9')
            {
                mBuffer.Read();
                next = mBuffer.Peek();
            }

            // Read the fractional part if there is one
            if (next == '.')
            {

                // Read the fractional part
                next = mBuffer.Peek();

                while (next >= '0' && next <= '9')
                {
                    mBuffer.Read();
                    next = mBuffer.Peek();
                }

                // Finish off the token
                this.SetTokenLiteralNumber(true);

            }
            else
            {

                // Finish off the token
                this.SetTokenLiteralNumber(false);
            }
        }

        private void ReadString()
        {
            int current = mBuffer.Read();
            var actualString = new StringBuilder();

            for (; ; )
            {
                if (current == '"')
                {
                    int next = mBuffer.Peek();

                    if (next == '"')
                    {
                        actualString.Append('"');
                        mBuffer.Read();
                    }
                    else
                    {
                        break;
                    }
                }
                else if (!IsValidStringChar(current))
                {

                    // Abort...
                    this.SetTokenUnexpectedCharacter(BasicTokenKind.LiteralString);
                    return;
                }
                else
                {
                    actualString.Append((char)current);
                }

                current = mBuffer.Read();
            }

            // Successfully read the string 
            SetTokenLiteralString(actualString.ToString(), true);
        }

        #endregion

        #region Private Methods - Token State

        private void SetTokenHelper_ClearLiteralData()
        {
            mCurrent.ValueString = null;
            mCurrent.ValueLong = 0L;
            mCurrent.ValueDouble = 0.0D;
        }

        private void SetTokenEof()
        {

            // Make sure all literal data gets reset
            this.SetTokenHelper_ClearLiteralData();

            // Mark the end of the token
            mBuffer.MarkSingleLineTokenEnd();

            // Set state
            mCurrent.Category = TokenCategory.EndOfStream;
            mCurrent.Kind = BasicTokenKind.EndOfFile;
            mCurrent.Span = SourceSpan.None;
        }

        private void SetTokenLiteralString(string actualString, bool isValid)
        {

            // Make sure all literal data gets reset
            this.SetTokenHelper_ClearLiteralData();

            // Mark the end of the token
            mBuffer.MarkSingleLineTokenEnd();

            // Set state
            mCurrent.Category = TokenCategory.StringLiteral;
            mCurrent.Kind = BasicTokenKind.LiteralString;
            mCurrent.Span = mBuffer.TokenSpan;
            mCurrent.ValueString = actualString;
        }

        private void SetTokenLiteralNumber(bool isFractional)
        {

            // Make sure all literal data gets reset
            this.SetTokenHelper_ClearLiteralData();

            // Setup the token and parse the number
            mBuffer.MarkSingleLineTokenEnd();
            mCurrent.Span = mBuffer.TokenSpan;
            mCurrent.ValueString = mBuffer.GetTokenString();
            mCurrent.Category = TokenCategory.NumericLiteral;

            if (isFractional)
            {
                double result;

                if (double.TryParse(mCurrent.ValueString, out result))
                {
                    mCurrent.ValueDouble = result;                    
                    mCurrent.Kind = BasicTokenKind.LiteralFloatingPointNumber;
                }
                else
                {
                    mCurrent.Kind = BasicTokenKind.InvalidNumber;
                }
            }
            else
            {
                long result;

                if (Int64.TryParse(mCurrent.ValueString, out result))
                {
                    mCurrent.ValueLong = result;
                    mCurrent.Kind = BasicTokenKind.LiteralInteger;
                }
                else
                {
                    mCurrent.Kind = BasicTokenKind.InvalidNumber;
                }
            }
        }

        private void SetTokenOperator(BasicTokenKind operatorKind)
        {

            // Make sure all literal data gets reset
            this.SetTokenHelper_ClearLiteralData();

            // Mark the end of the token
            mBuffer.MarkSingleLineTokenEnd();

            // Set state
            mCurrent.Kind = operatorKind;
            mCurrent.Span = mBuffer.TokenSpan;
            mCurrent.Category = TokenCategory.Operator;
        }

        private void SetTokenUnexpectedCharacter()
        {

        }

        private void SetTokenUnexpectedCharacter(BasicTokenKind expectedTokenKind)
        {

        }


        #endregion

        #region Private Methods

        private static bool IsIdentifierChar(int ch)
        {
            return char.IsLetterOrDigit((char)ch) || '_' == ch;
        }

        private static bool IsWhitespaceOrNewLine(int ch)
        {
            return IsWhitespace(ch) || IsNewLine(ch);
        }

        private static bool IsWhitespace(int ch)
        {
            return ch == ' ' || ch == '\t';
        }

        private static bool IsNewLine(int ch)
        {
            return ch == '\n' || ch == '\r';
        }

        private static bool IsValidStringChar(int ch)
        {
            return ch >= 0 && !char.IsControl((char)ch);
        }

        #endregion
    }
}
